From d1220c426cc0fd0a76d517d450cd280450315853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lestin=20Matte?= Date: Thu, 10 Aug 2023 11:08:03 +0200 Subject: [PATCH] Add search front-end from pgweb Add related variables in settings.py --- django/archives/mailarchives/decorators.py | 15 + django/archives/mailarchives/models.py | 4 + .../archives/mailarchives/templates/base.html | 2 +- .../mailarchives/templates/listsearch.html | 65 ++++ .../mailarchives/templates/searchform.html | 4 +- django/archives/mailarchives/views.py | 335 ++++++++++++++---- django/archives/settings.py | 8 +- django/archives/urls.py | 6 +- 8 files changed, 371 insertions(+), 68 deletions(-) create mode 100644 django/archives/mailarchives/decorators.py create mode 100644 django/archives/mailarchives/templates/listsearch.html diff --git a/django/archives/mailarchives/decorators.py b/django/archives/mailarchives/decorators.py new file mode 100644 index 0000000..bddc4a7 --- /dev/null +++ b/django/archives/mailarchives/decorators.py @@ -0,0 +1,15 @@ +import datetime +from functools import wraps +from collections import defaultdict +from django.contrib.auth.decorators import login_required as django_login_required + + +def queryparams(*args): + """ + Allow specified query parameters when calling function. + NOTE! Must be the "outermost" decorator!!! + """ + def _queryparams(fn): + fn.queryparams = args + return fn + return _queryparams diff --git a/django/archives/mailarchives/models.py b/django/archives/mailarchives/models.py index 44c4469..00fef56 100644 --- a/django/archives/mailarchives/models.py +++ b/django/archives/mailarchives/models.py @@ -77,6 +77,10 @@ class ListGroup(models.Model): class Meta: db_table = 'listgroups' + @property + def negid(self): + return -self.groupid + class List(models.Model): listid = models.IntegerField(null=False, primary_key=True) diff --git a/django/archives/mailarchives/templates/base.html b/django/archives/mailarchives/templates/base.html index 23d137e..ab2ea6d 100644 --- a/django/archives/mailarchives/templates/base.html +++ b/django/archives/mailarchives/templates/base.html @@ -53,7 +53,7 @@
  • Donate
  • Your account
    • -
    +
    diff --git a/django/archives/mailarchives/templates/listsearch.html b/django/archives/mailarchives/templates/listsearch.html new file mode 100644 index 0000000..4035e64 --- /dev/null +++ b/django/archives/mailarchives/templates/listsearch.html @@ -0,0 +1,65 @@ +{%extends "page.html"%} +{%block title%}Search results{%endblock%} +{%block contents%} + + + +
    +
    + +
    + + + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    +
    + + +{%if search_error %} +
    {{search_error}}
    +{%else%} + + {%if hitcount == 0 %} +

    Your search for {{query}} returned no hits.

    + {%else%} +

    Results {{firsthit}}-{{lasthit}} of {%if hitcount == 1000%}more than 1000{%else%}{{hitcount}}{%endif%}.

    + {%if pagelinks %}Result pages: {{pagelinks|safe}}

    {%endif%} + {%for hit in hits %} + {{forloop.counter0|add:firsthit}}. {{hit.subject}} [{{hit.rank|floatformat:2}}]
    + From {{hit.author}} on {{hit.date}}.
    + {{hit.abstract|safe}}
    + {{ archives_root }}/message-id/{{hit.messageid}}
    +
    + {%endfor%} + {%if pagelinks %}Result pages: {{pagelinks|safe}}

    {%endif%} + {%endif%} +{%endif%} +{%endblock%} diff --git a/django/archives/mailarchives/templates/searchform.html b/django/archives/mailarchives/templates/searchform.html index 1e5b4a2..4a1c6df 100644 --- a/django/archives/mailarchives/templates/searchform.html +++ b/django/archives/mailarchives/templates/searchform.html @@ -1,6 +1,6 @@

    Search the Archives

    -
    + {%if searchform_listname%} {%endif%}
    @@ -17,7 +17,7 @@ (enter a message-id to go directly to that message)
    - Advanced Search + Advanced Search
    diff --git a/django/archives/mailarchives/views.py b/django/archives/mailarchives/views.py index bf3336e..13a3d9a 100644 --- a/django/archives/mailarchives/views.py +++ b/django/archives/mailarchives/views.py @@ -1,5 +1,5 @@ from django.template import RequestContext -from django.http import HttpResponse, HttpResponseForbidden, Http404 +from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404 from django.http import StreamingHttpResponse, HttpResponseRedirect from django.http import HttpResponsePermanentRedirect, HttpResponseNotModified from django.core.exceptions import PermissionDenied @@ -19,16 +19,28 @@ import calendar import email.parser import email.policy from io import BytesIO -from urllib.parse import quote +from urllib.parse import quote, quote_plus, urlencode import ipaddress +import requests import json from .redirecthandler import ERedirect +from .decorators import queryparams + from .models import * +# Conditionally import memcached library. Everything will work without +# it, so we allow development installs to run without it... +try: + import pylibmc + has_memcached = True +except Exception as e: + has_memcached = False + + # Ensure the user is logged in (if it's not public lists) def ensure_logged_in(request): if settings.PUBLIC_ARCHIVES: @@ -175,7 +187,7 @@ class NavContext(object): self.request = request self.ctx = { 'allow_resend': settings.ALLOW_RESEND, - 'PGWEB_ADDRESS': settings.PGWEB_ADDRESS, + 'ARCHIVES_FRONT_ADDRESS': settings.ARCHIVES_FRONT_ADDRESS, } if all_groups: @@ -210,6 +222,7 @@ def index(request): (groups, listgroupid) = get_all_groups_and_lists(request) return render_nav(NavContext(request, all_groups=groups), 'index.html', { 'groups': [{'groupname': g['groupname'], 'lists': g['lists']} for g in groups], + 'ARCHIVES_FRONT_ADDRESS': settings.ARCHIVES_FRONT_ADDRESS, }) @@ -222,6 +235,7 @@ def groupindex(request, groupid): return render_nav(NavContext(request, all_groups=groups, expand_groupid=groupid), 'index.html', { 'groups': mygroups, + 'ARCHIVES_FRONT_ADDRESS': settings.ARCHIVES_FRONT_ADDRESS, }) @@ -237,6 +251,7 @@ def monthlist(request, listname): return render_nav(NavContext(request, l.listid, l.listname), 'monthlist.html', { 'list': l, 'months': months, + 'ARCHIVES_FRONT_ADDRESS': settings.ARCHIVES_FRONT_ADDRESS, }) @@ -288,6 +303,7 @@ def _render_datelist(request, l, d, datefilter, title, queryproc): 'title': title, 'daysinmonth': daysinmonth, 'yearmonth': yearmonth, + 'ARCHIVES_FRONT_ADDRESS': settings.ARCHIVES_FRONT_ADDRESS, }) if settings.PUBLIC_ARCHIVES: r['xkey'] = ' '.join(['pgam_{0}/{1}/{2}'.format(l.listid, year, month) for year, month in allyearmonths]) @@ -706,67 +722,60 @@ def resend_complete(request, messageid): }) -@csrf_exempt -def search(request): - if not settings.PUBLIC_ARCHIVES: - # We don't support searching of non-public archives at all at this point. - # XXX: room for future improvement - return HttpResponseForbidden('Not public archives') - - # Only certain hosts are allowed to call the search API - allowed = False - for ip_range in settings.SEARCH_CLIENTS: - if ipaddress.ip_address(request.META['REMOTE_ADDR']) in ipaddress.ip_network(ip_range): - allowed = True - break - if not allowed: - return HttpResponseForbidden('Invalid host') - - curs = connection.cursor() - - # Perform a search of the archives and return a JSON document. - # Expects the following (optional) POST parameters: - # q = query to search for - # ln = comma separate list of listnames to search in - # d = number of days back to search for, or -1 (or not specified) - # to search the full archives - # s = sort results by ['r'=rank, 'd'=date, 'i'=inverse date] - if not request.method == 'POST': - raise Http404('I only respond to POST') - - if 'q' not in request.POST: - raise Http404('No search query specified') - query = request.POST['q'] - - if 'ln' in request.POST: +def search_params(params): + lists = None + listid = None + if 'd' in params: try: - curs.execute("SELECT listid FROM lists WHERE listname=ANY(%(names)s)", { - 'names': request.POST['ln'].split(','), - }) - lists = [x for x, in curs.fetchall()] - except Exception: - # If failing to parse list of lists, just search all - lists = None + days = int(params['d']) + except Exception as e: + days = 365 else: - lists = None + days = 365 - if 'd' in request.POST: - days = int(request.POST['d']) - if days < 1 or days > 365: - firstdate = None - else: - firstdate = datetime.now() - timedelta(days=days) + if 'l' in params and params['l'] != '': + try: + listid = int(params['l']) + if listid < 0: + # This is a list group, we expand that on the web server + # Negative means it's a group, so verify that it exists + if not List.objects.filter(group=-listid).exists(): + raise Http404() + lists = '{{{0}}}'.format(','.join([str(x.listid) for x in List.objects.filter(group=-listid)])) + else: + # Make sure the list exists + if not List.objects.filter(pk=listid).exists(): + raise Http404() + lists = '{{{0}}}'.format(List.objects.get(pk=listid).listid) + except ValueError: + # If it's not an integer we just don't care + listid = None else: - firstdate = None + # Listid not specified. But do we have the name? + if 'ln' in params: + try: + ll = List.objects.get(listname=params['ln']) + listid = ll.listid + lists = '{{{0}}}'.format(listid) + except List.DoesNotExist: + # Invalid list name just resets the default of the form, + # no need to throw an error. + listid = None + else: + listid = None - if 's' in request.POST: - list_sort = request.POST['s'] - if list_sort not in ('d', 'r', 'i'): - list_stort = 'r' + if 's' in params: + listsort = params['s'] + if listsort not in ('r', 'd', 'i'): + listsort = 'r' else: - list_sort = 'r' + listsort = 'r' - # Ok, we have all we need to do the search + return days, listid, lists, listsort + + +def perform_search(query, days, lists, list_sort): + curs = connection.cursor() if query.find('@') > 0: # This could be a messageid. So try to get that one specifically first. @@ -789,6 +798,12 @@ def search(request): params = { 'q': query, } + + if days is None or days < 1 or days > 365: + firstdate = None + else: + firstdate = datetime.now() - timedelta(days=days) + if lists: qstr += " AND EXISTS (SELECT 1 FROM list_threads lt WHERE lt.threadid=m.threadid AND lt.listid=ANY(%(lists)s))" params['lists'] = lists @@ -804,9 +819,7 @@ def search(request): curs.execute(qstr, params) - resp = HttpResponse(content_type='application/json') - - json.dump([ + results = [ { 'm': messageid, 'd': date.isoformat(), @@ -814,11 +827,183 @@ def search(request): 'f': mailfrom, 'r': rank, 'a': abstract.replace("[[[[[[", "").replace("]]]]]]", ""), - } for messageid, date, subject, mailfrom, rank, abstract in curs.fetchall()], - resp) + } for messageid, date, subject, mailfrom, rank, abstract in curs.fetchall() + ] + + return results + + +@csrf_exempt +def search_api(request): + if not settings.PUBLIC_ARCHIVES: + # We don't support searching of non-public archives at all at this point. + # XXX: room for future improvement + return HttpResponseForbidden('Not public archives') + + # Only certain hosts are allowed to call the search API + allowed = False + for ip_range in settings.SEARCH_CLIENTS: + if ipaddress.ip_address(request.META['REMOTE_ADDR']) in ipaddress.ip_network(ip_range): + allowed = True + break + if not allowed: + return HttpResponseForbidden('Invalid host') + + curs = connection.cursor() + + # Perform a search of the archives and return a JSON document. + # Expects the following (optional) POST parameters: + # q = query to search for + # ln = comma separate list of listnames to search in + # d = number of days back to search for, or -1 (or not specified) + # to search the full archives + # s = sort results by ['r'=rank, 'd'=date, 'i'=inverse date] + if not request.method == 'POST': + raise Http404('I only respond to POST') + + if 'q' not in request.POST: + raise Http404('No search query specified') + + query = request.POST['q'] + + days, listid, lists, listsort = search_params(request.POST) + + # Ok, we have all we need to do the search + results = perform_search(query, days, lists, list_sort) + resp = HttpResponse(content_type='application/json') + + json.dump(results, resp) + return resp +@csrf_exempt +@queryparams('d', 'l', 'ln', 'p', 'q', 's', 'u') +@cache(minutes=30) +def search(request): + + if not settings.USE_SEARCH_FRONTEND: + raise Http404() + + if not settings.PUBLIC_ARCHIVES: + # We don't support searching of non-public archives at all at this point. + # XXX: room for future improvement + return HttpResponseForbidden('Not public archives') + + # constants that we might eventually want to make configurable + hitsperpage = 20 + + dateval, listid, lists, listsort = search_params(request.GET) + + sortoptions = ( + {'val': 'r', 'text': 'Rank', 'selected': request.GET.get('s', '') not in ('d', 'i')}, + {'val': 'd', 'text': 'Date', 'selected': request.GET.get('s', '') == 'd'}, + {'val': 'i', 'text': 'Reverse date', 'selected': request.GET.get('s', '') == 'i'}, + ) + dateoptions = ( + {'val': -1, 'text': 'anytime'}, + {'val': 1, 'text': 'within last day'}, + {'val': 7, 'text': 'within last week'}, + {'val': 31, 'text': 'within last month'}, + {'val': 186, 'text': 'within last 6 months'}, + {'val': 365, 'text': 'within last year'}, + ) + + # Check that we actually have something to search for + if request.GET.get('q', '') == '': + return render(request, 'listsearch.html', { + 'search_error': "No search term specified.", + 'sortoptions': sortoptions, + 'lists': List.objects.all().order_by("group__sortkey"), + 'listid': listid, + 'dates': dateoptions, + 'dateval': dateval, + 'archives_root': settings.ARCHIVES_FRONT_ADDRESS, + }) + + # Is the request being paged? + try: + pagenum = int(request.GET.get('p', 1)) + except Exception as e: + pagenum = 1 + + firsthit = (pagenum - 1) * hitsperpage + 1 + + query = request.GET['q'].strip() + # Lists are searched by passing the work down using a http + # API. In the future, we probably want to do everything + # through a http API and merge hits, but that's for later + p = { + 'q': query.encode('utf-8'), + 's': listsort, + 'ln': lists, + 'd': dateval, + } + urlstr = urlencode(p) + + # If memcached is available, let's try it + hits = None + if has_memcached: + memc = pylibmc.Client(['127.0.0.1', ], binary=True) + # behavior not supported on pylibmc in squeeze:: behaviors={'tcp_nodelay':True}) + try: + hits = memc.get(urlstr) + except Exception: + # If we had an exception, don't try to store either + memc = None + if not hits: + # No hits found - so try to get them from the search server + + hits = perform_search(query, dateval, lists, listsort) + + if has_memcached and memc: + # Store them in memcached too! But only for 10 minutes... + # And always compress it, just because we can + memc.set(urlstr, hits, 60 * 10, 1) + memc = None + lists = List.objects.get(pk=listid).listname + + if isinstance(hits, dict): + # This is not just a list of hits. + # Right now the only supported dict result is a messageid + # match, but make sure that's what it is. + if hits['messageidmatch'] == 1: + return HttpResponseRedirect("/message-id/%s" % query) + + totalhits = len(hits) + querystr = "?m=1&q=%s&l=%s&d=%s&s=%s" % ( + quote_plus(query.encode('utf-8')), + listid or '', + dateval, + listsort + ) + + return render(request, 'listsearch.html', { + 'hitcount': totalhits, + 'firsthit': firsthit, + 'lasthit': min(totalhits, firsthit + hitsperpage - 1), + 'query': request.GET['q'], + 'archives_root': settings.ARCHIVES_FRONT_ADDRESS, + 'pagelinks': " ".join( + generate_pagelinks(pagenum, + (totalhits - 1) // hitsperpage + 1, + querystr)), + 'hits': [{ + 'date': h['d'], + 'subject': h['s'], + 'author': h['f'], + 'messageid': h['m'], + 'abstract': h['a'], + 'rank': h['r'], + } for h in hits[firsthit - 1:firsthit + hitsperpage - 1]], + 'sortoptions': sortoptions, + 'lists': List.objects.all().order_by("group__sortkey"), + 'listid': listid, + 'dates': dateoptions, + 'dateval': dateval, + }) + + @cache(seconds=10) def web_sync_timestamp(request): s = datetime.now().strftime("%Y-%m-%d %H:%M:%S\n") @@ -909,3 +1094,29 @@ def slash_redirect(request, url): @cache(hours=8) def re_redirect(request, prefix, msgid): return HttpResponsePermanentRedirect("/%s%s" % (prefix, msgid)) + + +def generate_pagelinks(pagenum, totalpages, querystring): + # Generate a list of links to page through a search result + # We generate these in HTML from the python code because it's + # simply too ugly to try to do it in the template. + if totalpages < 2: + return + + if pagenum > 1: + # Prev link + yield 'Prev' % (querystring, pagenum - 1) + + if pagenum > 10: + start = pagenum - 10 + else: + start = 1 + + for i in range(start, min(start + 20, totalpages + 1)): + if i == pagenum: + yield "%s" % i + else: + yield '%s' % (querystring, i, i) + + if pagenum != min(start + 20, totalpages): + yield 'Next' % (querystring, pagenum + 1) diff --git a/django/archives/settings.py b/django/archives/settings.py index 24861a9..b07f85c 100644 --- a/django/archives/settings.py +++ b/django/archives/settings.py @@ -148,7 +148,10 @@ PGAUTH_REDIRECT = "http://localhost:8000/account/auth/12/" PGAUTH_KEY = "encryption_key" ALLOW_RESEND = False -PGWEB_ADDRESS = 'https://www.postgresql.org' +USE_SEARCH_FRONTEND = True +ARCHIVES_FRONT_ADDRESS = 'https://www.postgresql.org' +ARCHIVES_SEARCH_PLAINTEXT = True +ARCHIVES_SEARCH_SERVER = 'localhost' try: from .settings_local import * @@ -175,3 +178,6 @@ if ALLOW_RESEND or not PUBLIC_ARCHIVES: if not PUBLIC_ARCHIVES: from archives.util import validate_new_user PGAUTH_CREATEUSER_CALLBACK = validate_new_user + +if USE_SEARCH_FRONTEND: + ARCHIVES_FRONT_ADDRESS = '' diff --git a/django/archives/urls.py b/django/archives/urls.py index bc8a18d..2dfab30 100644 --- a/django/archives/urls.py +++ b/django/archives/urls.py @@ -42,8 +42,10 @@ urlpatterns = [ url(r'^list/([\w-]+)/mbox/([\w-]+)\.(\d{4})(\d{2})', archives.mailarchives.views.mbox), - # Search - url(r'^archives-search/', archives.mailarchives.views.search), + # Search API + url(r'^archives-search/', archives.mailarchives.views.search_api), + # Search front-end + url(r'^search/$', archives.mailarchives.views.search), # Date etc indexes url(r'^list/([\w-]+)/$', archives.mailarchives.views.monthlist), -- 2.41.0